{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "24ae5547",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3395870c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-13 15:24:45.060047: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:45.064488: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:45.065224: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:45.113874: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-07-13 15:24:45.114686: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:45.115450: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:45.116166: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:47.816542: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:47.817225: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:47.817884: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-13 15:24:47.818507: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2022-07-13 15:24:47.818528: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 20198 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/apex/pyprof/__init__.py:5: FutureWarning: pyprof will be removed by the end of June, 2022\n",
      "  warnings.warn(\"pyprof will be removed by the end of June, 2022\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from transformers import T5Config, TFT5Model, T5Model, load_tf_weights_in_t5, T5Tokenizer\n",
    "from transformers import T5ForConditionalGeneration, TFT5ForConditionalGeneration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "51fbf230",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a541b0ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "out = 'temp'\n",
    "os.makedirs(out, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "48753c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = T5Config.from_pretrained('malay-huggingface/t5-tiny-bahasa-cased')\n",
    "config.save_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "50a47b1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint\r\n",
      "events.out.tfevents.1657085426.huseincomel-desktop\r\n",
      "events.out.tfevents.1657095770.huseincomel-desktop\r\n",
      "events.out.tfevents.1657207150.huseincomel-desktop\r\n",
      "events.out.tfevents.1657554594.huseincomel-desktop\r\n",
      "events.out.tfevents.1657595338.huseincomel-desktop\r\n",
      "events.out.tfevents.1657622736.huseincomel-desktop\r\n",
      "graph.pbtxt\r\n",
      "model.ckpt-1031000.data-00000-of-00002\r\n",
      "model.ckpt-1031000.data-00001-of-00002\r\n",
      "model.ckpt-1031000.index\r\n",
      "model.ckpt-1031000.meta\r\n",
      "model.ckpt-1073000.data-00000-of-00002\r\n",
      "model.ckpt-1073000.data-00001-of-00002\r\n",
      "model.ckpt-1073000.index\r\n",
      "model.ckpt-1073000.meta\r\n",
      "model.ckpt-1103000.data-00000-of-00002\r\n",
      "model.ckpt-1103000.data-00001-of-00002\r\n",
      "model.ckpt-1103000.index\r\n",
      "model.ckpt-1103000.meta\r\n",
      "model.ckpt-1132000.data-00000-of-00002\r\n",
      "model.ckpt-1132000.data-00001-of-00002\r\n",
      "model.ckpt-1132000.index\r\n",
      "model.ckpt-1132000.meta\r\n",
      "model.ckpt-1158000.data-00000-of-00002\r\n",
      "model.ckpt-1158000.data-00001-of-00002\r\n",
      "model.ckpt-1158000.index\r\n",
      "model.ckpt-1158000.meta\r\n",
      "model.ckpt-1188000.data-00000-of-00002\r\n",
      "model.ckpt-1188000.data-00001-of-00002\r\n",
      "model.ckpt-1188000.index\r\n",
      "model.ckpt-1188000.meta\r\n",
      "model.ckpt-1219000.data-00000-of-00002\r\n",
      "model.ckpt-1219000.data-00001-of-00002\r\n",
      "model.ckpt-1219000.index\r\n",
      "model.ckpt-1219000.meta\r\n",
      "model.ckpt-1249000.data-00000-of-00002\r\n",
      "model.ckpt-1249000.data-00001-of-00002\r\n",
      "model.ckpt-1249000.index\r\n",
      "model.ckpt-1249000.meta\r\n",
      "model.ckpt-1279000.data-00000-of-00002\r\n",
      "model.ckpt-1279000.data-00001-of-00002\r\n",
      "model.ckpt-1279000.index\r\n",
      "model.ckpt-1279000.meta\r\n",
      "model.ckpt-1309000.data-00000-of-00002\r\n",
      "model.ckpt-1309000.data-00001-of-00002\r\n",
      "model.ckpt-1309000.index\r\n",
      "model.ckpt-1309000.meta\r\n",
      "model.ckpt-1339000.data-00000-of-00002\r\n",
      "model.ckpt-1339000.data-00001-of-00002\r\n",
      "model.ckpt-1339000.index\r\n",
      "model.ckpt-1339000.meta\r\n",
      "model.ckpt-1369000.data-00000-of-00002\r\n",
      "model.ckpt-1369000.data-00001-of-00002\r\n",
      "model.ckpt-1369000.index\r\n",
      "model.ckpt-1369000.meta\r\n",
      "model.ckpt-1432000.data-00000-of-00002\r\n",
      "model.ckpt-1432000.data-00001-of-00002\r\n",
      "model.ckpt-1432000.index\r\n",
      "model.ckpt-1432000.meta\r\n",
      "model.ckpt-1463000.data-00000-of-00002\r\n",
      "model.ckpt-1463000.data-00001-of-00002\r\n",
      "model.ckpt-1463000.index\r\n",
      "model.ckpt-1463000.meta\r\n",
      "model.ckpt-1495000.data-00000-of-00002\r\n",
      "model.ckpt-1495000.data-00001-of-00002\r\n",
      "model.ckpt-1495000.index\r\n",
      "model.ckpt-1495000.meta\r\n",
      "model.ckpt-1532000.data-00000-of-00002\r\n",
      "model.ckpt-1532000.data-00001-of-00002\r\n",
      "model.ckpt-1532000.index\r\n",
      "model.ckpt-1532000.meta\r\n",
      "model.ckpt-1564000.data-00000-of-00002\r\n",
      "model.ckpt-1564000.data-00001-of-00002\r\n",
      "model.ckpt-1564000.index\r\n",
      "model.ckpt-1564000.meta\r\n",
      "model.ckpt-1596000.data-00000-of-00002\r\n",
      "model.ckpt-1596000.data-00001-of-00002\r\n",
      "model.ckpt-1596000.index\r\n",
      "model.ckpt-1596000.meta\r\n",
      "model.ckpt-1628000.data-00000-of-00002\r\n",
      "model.ckpt-1628000.data-00001-of-00002\r\n",
      "model.ckpt-1628000.index\r\n",
      "model.ckpt-1628000.meta\r\n",
      "model.ckpt-1660000.data-00000-of-00002\r\n",
      "model.ckpt-1660000.data-00001-of-00002\r\n",
      "model.ckpt-1660000.index\r\n",
      "model.ckpt-1660000.meta\r\n",
      "model.ckpt-1720000.data-00000-of-00002\r\n",
      "model.ckpt-1720000.data-00001-of-00002\r\n",
      "model.ckpt-1720000.index\r\n",
      "model.ckpt-1720000.meta\r\n",
      "model.ckpt-1830000.data-00000-of-00002\r\n",
      "model.ckpt-1830000.data-00001-of-00002\r\n",
      "model.ckpt-1830000.index\r\n",
      "model.ckpt-1830000.meta\r\n",
      "model.ckpt-1860000.data-00000-of-00002\r\n",
      "model.ckpt-1860000.data-00001-of-00002\r\n",
      "model.ckpt-1860000.index\r\n",
      "model.ckpt-1860000.meta\r\n",
      "model.ckpt-1870000.data-00000-of-00002\r\n",
      "model.ckpt-1870000.data-00001-of-00002\r\n",
      "model.ckpt-1870000.index\r\n",
      "model.ckpt-1870000.meta\r\n",
      "model.ckpt-1880000.data-00000-of-00002\r\n",
      "model.ckpt-1880000.data-00001-of-00002\r\n",
      "model.ckpt-1880000.index\r\n",
      "model.ckpt-1880000.meta\r\n",
      "model.ckpt-1890000.data-00000-of-00002\r\n",
      "model.ckpt-1890000.data-00001-of-00002\r\n",
      "model.ckpt-1890000.index\r\n",
      "model.ckpt-1890000.meta\r\n",
      "model.ckpt-1900000.data-00000-of-00002\r\n",
      "model.ckpt-1900000.data-00001-of-00002\r\n",
      "model.ckpt-1900000.index\r\n",
      "model.ckpt-1900000.meta\r\n",
      "operative_config.gin\r\n"
     ]
    }
   ],
   "source": [
    "!ls t5-tiny-noisy-ms-en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f54394ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T5Model(\n",
       "  (shared): Embedding(32128, 384)\n",
       "  (encoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = T5Model(config)\n",
    "load_tf_weights_in_t5(model, config, 't5-tiny-noisy-ms-en/model.ckpt-1900000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "faf936d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !mkdir temp-t5-tiny\n",
    "# !cp t5-tiny-noisy-ms-en/model.ckpt-1361000* temp-t5-tiny"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "86b9d748",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer('sp10m.cased.ms-en.model', padding = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7e68dcc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "737e7c80",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gen = T5ForConditionalGeneration.from_pretrained(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "005b412a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-13 15:24:50.407393: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n",
      "2022-07-13 15:24:50.870924: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFT5ForConditionalGeneration: ['decoder.embed_tokens.weight', 'encoder.embed_tokens.weight']\n",
      "- This IS expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFT5ForConditionalGeneration from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFT5ForConditionalGeneration were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFT5ForConditionalGeneration for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "model_tf = TFT5ForConditionalGeneration.from_pretrained(out, from_pt = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fc1a66fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ms-en-right.test') as fopen:\n",
    "    right = fopen.read().split('\\n')\n",
    "    \n",
    "with open('ms-en-left.test') as fopen:\n",
    "    left = fopen.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3124e42d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T5ForConditionalGeneration(\n",
       "  (shared): Embedding(32128, 384)\n",
       "  (encoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder): T5Stack(\n",
       "    (embed_tokens): Embedding(32128, 384)\n",
       "    (block): ModuleList(\n",
       "      (0): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "              (relative_attention_bias): Embedding(32, 12)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): T5Block(\n",
       "        (layer): ModuleList(\n",
       "          (0): T5LayerSelfAttention(\n",
       "            (SelfAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): T5LayerCrossAttention(\n",
       "            (EncDecAttention): T5Attention(\n",
       "              (q): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (k): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (v): Linear(in_features=384, out_features=768, bias=False)\n",
       "              (o): Linear(in_features=768, out_features=384, bias=False)\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): T5LayerFF(\n",
       "            (DenseReluDense): T5DenseReluDense(\n",
       "              (wi): Linear(in_features=384, out_features=1344, bias=False)\n",
       "              (wo): Linear(in_features=1344, out_features=384, bias=False)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "              (relu_act): ReLU()\n",
       "            )\n",
       "            (layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layer_norm): FusedRMSNorm(torch.Size([384]), eps=1e-06, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=384, out_features=32128, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gen.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "35abd80e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████| 5475/5475 [32:45<00:00,  2.79it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "batch_size = 16\n",
    "\n",
    "results = []\n",
    "for i in tqdm(range(0, len(left), batch_size)):\n",
    "    input_ids = [{'input_ids': tokenizer.encode(f'terjemah Melayu ke Inggeris: {s}', return_tensors = 'pt')[0]} for s in left[i:i + batch_size]]\n",
    "    padded = tokenizer.pad(input_ids, padding = 'longest')\n",
    "    outputs = model_gen.generate(padded['input_ids'].cuda(), attention_mask = padded['attention_mask'].cuda(), max_length = 1000)\n",
    "    # outputs = model_gen.generate(**padded, max_length = 1000)\n",
    "    for o in outputs:\n",
    "        results.append(tokenizer.decode(o, skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "eb550d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
    "\n",
    "bleu = BLEU()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "20530f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_left, filtered_right = [], []\n",
    "for no, r in enumerate(results):\n",
    "    if len(r):\n",
    "        filtered_left.append(r)\n",
    "        filtered_right.append(right[no])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e50b3fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "refs = [filtered_right]\n",
    "sys = filtered_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e5d6f81d",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = bleu.corpus_score(sys, refs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cf03103f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'BLEU',\n",
       " 'score': 65.9069151371865,\n",
       " '_mean': -1.0,\n",
       " '_ci': -1.0,\n",
       " '_verbose': '83.0/69.3/60.7/54.1 (BP = 1.000 ratio = 1.001 hyp_len = 2003273 ref_len = 2001100)',\n",
       " 'bp': 1.0,\n",
       " 'counts': [1662910, 1327225, 1108852, 941870],\n",
       " 'totals': [2003273, 1915678, 1828247, 1741231],\n",
       " 'sys_len': 2003273,\n",
       " 'ref_len': 2001100,\n",
       " 'precisions': [83.00965470008332,\n",
       "  69.28225933585915,\n",
       "  60.651104582695886,\n",
       "  54.09219109928551],\n",
       " 'prec_str': '83.0/69.3/60.7/54.1',\n",
       " 'ratio': 1.0010859027534855}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "88704b13",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.push_to_hub('t5-tiny-finetuned-noisy-ms-en', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "97afc0d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04b002d82e274dd5b3e3e121447a2ae2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 4.00k/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-ms-en\n",
      "   cc98a1d..bf3b4b9  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-ms-en/commit/bf3b4b9b6fe7df7a21152bb2b6c509476ab8056e'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gen.push_to_hub('t5-tiny-finetuned-noisy-ms-en', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "db817c07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17a423d1053a4597990c7a11fc708c5e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file tf_model.h5:   0%|          | 4.00k/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-ms-en\n",
      "   bf3b4b9..f4b5b53  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-ms-en/commit/f4b5b5321138db069e7404b80cd44eb00d250823'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_tf.push_to_hub('t5-tiny-finetuned-noisy-ms-en', organization='mesolitica',\n",
    "                    tags = 'generated_from_keras_callback_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "2819f991",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Already up to date.\r\n"
     ]
    }
   ],
   "source": [
    "!cd t5-tiny-finetuned-noisy-ms-en && git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "df9b8117",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[main 616ef8f] add tensorboard\n",
      " 2 files changed, 5 insertions(+), 2 deletions(-)\n",
      " create mode 100644 events.out.tfevents.1657622736.huseincomel-desktop\n",
      "Uploading LFS objects: 100% (2/2), 77 MB | 4.2 MB/s, done.                      \n",
      "Enumerating objects: 6, done.\n",
      "Counting objects: 100% (6/6), done.\n",
      "Delta compression using up to 16 threads\n",
      "Compressing objects: 100% (4/4), done.\n",
      "Writing objects: 100% (4/4), 549 bytes | 549.00 KiB/s, done.\n",
      "Total 4 (delta 1), reused 0 (delta 0)\n",
      "To https://huggingface.co/mesolitica/t5-tiny-finetuned-noisy-ms-en\n",
      "   f4b5b53..616ef8f  main -> main\n"
     ]
    }
   ],
   "source": [
    "!cp t5-tiny-noisy-ms-en/*.tfevents.* t5-tiny-finetuned-noisy-ms-en\n",
    "!cd t5-tiny-finetuned-noisy-ms-en && git add . && git commit -m 'add tensorboard' && git push"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
